/*
******************
* author: 马丰彦
* date: 2019/1/14
* function: 爬取高匿IP池
******************
*/

var Crawler = require('crawler'); // node-crawler
var Ip = require('../model/proxyIp');
require('../connect.js');
var client = require('./redis');
client.del('list');
var ips = []
var proxyApi = 'https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list'
var proxy = new Crawler({
  maxConnections: 1,
  jQuery: false,
  callback: async (error, res, done) => {
    if (error) {
      console.log(error);
    } else {
      var reg = new RegExp('}', 'g'); // 为了分割字符串
      var data = res.body.replace(reg, '}!!');
      var reg1 = new RegExp('\\n', 'g');
      var last = data.replace(reg1, '')
      var ips = last.split('!!');
      ips.pop();
      var allIp = [];
      ips.forEach(function(item) {
        if (!!item) {
          var parse = JSON.parse(item);
          if (parse.country === 'CN') {
            client.rpush('list', item);
            allIp.push(parse); // 存入中国ip
          }
        }
      })
      await client.lrange('list', 0,-1, function (err, v) {
        ips = v;
      })
      Ip.insertMany(allIp, function(error, docs) {
        if (error) {
          console.log(error);
        } else {
          console.log('成功!');
        }
      })
    }
    done();
  }
})
proxy.queue(proxyApi);
module.exports = proxy
